"""
This module provides log writing and report generation capabilities to
the CCHR conversion software.
Public Classes:
Log -- Class for working with log files.
Report -- Class for generating and updating report information.
Exceptions:
LogError -- Exception raised when problem occurs in log manipulation.
ReportError -- Exception raised when error occurs in Report generation.
"""
from os import path, listdir, remove, mkdir
from datetime import datetime
import cardsharp as cs
from MySQLdb import connect
from errors import *
import re
import Queue
from util import RUNDATETIME, PHASE_MAP
from collections import defaultdict
__all__ = ['Log']
#TODO: add modifying this list to user interface
#list of log files, used in log.delete() to assert deletion of only log files
_log_files = [
'ncic.txt', 'ainc.txt', 'achgdetx.txt', 'acnt.txt', 'achgsvr.txt',
'adisp.txt', 'astate.txt','arrdatex.txt', 'asource.txt',
'cnic.txt', 'cchgdet.txt', 'ccnt.txt', 'cchgsvr.txt',
'cdsp.txt', 'cdispdatex.txt', 'snttext.txt', 'cstate.txt',
'csource.txt', 'sntdatex.txt',
'dctz2.txt', 'dpob2.txt', 'dsource.txt', 'dstatex.txt', 'gender.txt',
'race.txt',
'rapstate.txt'
]
#mapping of log.type to (name, output Variables)
_log_types = {0 : ('default', [('segment', 'integer'), 'variable',
('phase', 'integer'), ('id', 'integer'), 'convert_state',
'convert_key', 'error_message', 'convert_time']),
3 : ('cnt', [('segment', 'integer'), 'variable',
('phase', 'integer'), ('id', 'integer'), 'convert_state',
'convert_key', 'extracted_cts', 'error_message', 'convert_time']),
2 : ('offense', [('segment', 'integer'), 'variable',
('phase', 'integer'), ('id', 'integer'), 'convert_state', 'convert_key_names',
'convert_key_values', 'predictions', 'error_message', 'convert_time']),
1 : ('fed', [('segment', 'integer'), 'variable',
('phase', 'integer'), ('id', 'integer'), 'rapstatex',
('convert_state', 'integer'), 'ori', 'arrname', ('fed', 'integer'), 'convert_time']),
77 : ('report', [('id', 'integer'), 'region', 'rapstate', 'var_name', 'orig_value', 'new_value']),
78 : ('summary_report', [('casenum', 'integer'), 'var_name', 'orig_value', 'new_value']),
90 : ('ref_integ_error', ['segment', 'variable', 'phase', 'bad_value', 'table_name', 'allowed_values', 'rule_loc', 'error_message']),
91 : ('regex_init', ['message', 'filename', 'regex_pattern', 'regex_result', 'regex-priority', 'add_info_1', 'add_info_2', 'add_info_3']),
99 : ('fatal_error', ['segment', 'variable', 'phase', 'error_message', 'time'])
}
#TODO add setters for segment, phase, var to allow log creation outside of vc
[docs]class Log(object):
"""A class to support log file manipulation for CCHR data conversion
software.
Public Methods:
write -- Writes one line of log data to a log file.
load -- Loads log data into an internal cardsharp dataset object.
delete -- Deletes the log file.
"""
def __init__(self, **kw):
"""Initialize a new log object.
Keyword Arguments:
log_dir -- Path to the log file directory. **optional**
time -- A timestamp for the log, if not specified will
be automatically generated. **optional**
segment -- The segment id (integer) corresponding to the segment that
the log object is mapped to. **optional**
var -- The variable id (integer) corresponding to the variable that
the log object is mapped to. **optional**
phase -- The phase in the data conversion process which the
the log object is mapped to. **optional**
verbose -- Set to true for more detailed output. **optional**
"""
self.time = kw.get('time', datetime.now().isoformat('_').replace(':', '-'))
self.segment = kw.get('segment')
self.phase = kw.get('phase')
self.log_dir = kw.get('log_dir')
self.var = kw.get('var')
self.verbose = kw.get('verbose')
self.stored = Queue.Queue()
[docs] def load(self, fn):
"""Load a log file. The variables of the log file are determined by _log_types.
The fn needs to have a log type indicator before the .txt in the filename (fn).
Example) ancic_0.txt
@param fn: The filename of the log to load.
@return: (log_data, log_type) -- Returns tuple with the first value being a cardsharp dataset containing log data
and the second value is the log type.
"""
try:
type = re.search('(?<=_)\d+(?=.txt)', fn).group()
log_data = cs.load(source=path.join(self.log_dir, fn), format='text', var_names=_log_types[int(type)][1])
return (log_data, int(type))
except:
raise LogError('Can not load: %s' % fn)
[docs] def write(self, fn, vals, type=0, no_add_info=False, store=False):
"""Writes a line of log data to the log file.
@param fn: The name of log file to write (append) to.
@param vals: The list of the values to write to the log file.
@param type: The type of log to write. Default is 0 (see _log_types for list of avaialbe log types).
@param store: Whether to store the file in memory (can be dumped to file later) or to write to file immediately
"""
try:
if store:
if not no_add_info:
self.stored.put((fn,type,'\t'.join([str(self.segment), str(self.var), str(self.phase),
'\t'.join(str(val) for val in vals), '%s\n' % self.time])))
else:
self.stored.put((fn,type,'\t'.join(['\t'.join(str(val) for val in vals), '%s\n' % self.time])))
else:
with open(path.join(self.log_dir, '%s_%s.txt' % (fn, type)), 'a') as out:
if not no_add_info:
out.write('\t'.join([str(self.segment), str(self.var), str(self.phase),
'\t'.join(str(val) for val in vals), '%s\n' % self.time]))
else:
out.write('\t'.join(['\t'.join(str(val) for val in vals), '%s\n' % self.time]))
except:
raise
raise LogError('Failed to write to log.')
[docs] def dump(self):
"""Empties the log queue and writes to all of the log files for which there
is stored log output
"""
logs = defaultdict(list)
while not self.stored.empty():
temp = self.stored.get()
logs[(temp[0],temp[1])].append(temp[2])
for log in logs.keys():
for row in logs[log]:
with open(path.join(self.log_dir, '%s_%s.txt' % (log[0], log[1])), 'a') as out:
out.write(row)
[docs] def write_compare(self, fn, vals, type=0):
"""Writes a line of log data to the log file.
@param fn: The name of log file to write (append) to.
@param vals: The list of the values to write to the log file.
@param type: The type of log to write. Default is 0 (see _log_types for list of avaialbe log types).
"""
try:
with open(path.join(self.log_dir, '%s_%s.txt' % (fn, type)), 'a') as out:
out.write('\t'.join([str(val) for val in vals]) + '\n')
except:
raise LogError('Failed to write compare log.')
[docs] def delete(self, fn):
"""Deletes a log file.
Arguments:
fn -- name of the desired log file to delete
"""
try:
assert(fn in _log_files)
remove(path.join(self.log_dir, fn))
except AssertionError:
raise LogError('Unable to delete %s. Not listed as log file.' % fn)
except:
raise LogError('Unable to delete %s.' % fn)